<html><!-- Created using the cpp_pretty_printer from the dlib C++ library.  See http://dlib.net for updates. --><head><title>dlib C++ Library - dnn_self_supervised_learning_ex.cpp</title></head><body bgcolor='white'><pre>
<font color='#009900'>// The contents of this file are in the public domain. See LICENSE_FOR_EXAMPLE_PROGRAMS.txt
</font><font color='#009900'>/*
    This is an example illustrating the use of the deep learning tools from the dlib C++
    Library.  I'm assuming you have already read the <a href="dnn_introduction_ex.cpp.html">dnn_introduction_ex.cpp</a>, the
    <a href="dnn_introduction2_ex.cpp.html">dnn_introduction2_ex.cpp</a> and the <a href="dnn_introduction3_ex.cpp.html">dnn_introduction3_ex.cpp</a> examples.  In this example
    program we are going to show how one can train a neural network using an unsupervised
    loss function.  In particular, we will train the ResNet50 model from the paper
    "Deep Residual Learning for Image Recognition" by Kaiming He, Xiangyu Zhang, Shaoqing
    Ren, Jian Sun.

    To train the unsupervised loss, we will use the self-supervised learning (SSL) method
    called Barlow Twins, introduced in this paper:
    "Barlow Twins: Self-Supervised Learning via Redundancy Reduction" by Jure Zbontar,
    Li Jing, Ishan Misra, Yann LeCun, Stéphane Deny.

    The paper contains a good explanation on how and why this works, but the main idea
    behind the Barlow Twins method is:
        - generate two distorted views of a batch of images: YA, YB
        - feed them to a deep neural network and obtain their representations and
          and batch normalize them: ZA, ZB
        - compute the empirical cross-correlation matrix between both feature
          representations as: C = trans(ZA) * ZB.
        - make C as close as possible to the identity matrix.

    This removes the redundancy of the feature representations, by maximizing the
    encoded information about the images themselves, while minimizing the information
    about the transforms and data augmentations used to obtain the representations.

    The original Barlow Twins paper uses the ImageNet dataset, but in this example we
    are using CIFAR-10, so we will follow the recommendations of this paper, instead:
    "A Note on Connecting Barlow Twins with Negative-Sample-Free Contrastive Learning"
    by Yao-Hung Hubert Tsai, Shaojie Bai, Louis-Philippe Morency, Ruslan Salakhutdinov,
    in which they experiment with Barlow Twins on CIFAR-10 and Tiny ImageNet.  Since
    the CIFAR-10 contains relatively small images, we will define a ResNet50 architecture
    that doesn't downsample the input in the first convolutional layer, and doesn't have
    a max pooling layer afterwards, like the paper does.
*/</font>

<font color='#0000FF'>#include</font> <font color='#5555FF'>&lt;</font>dlib<font color='#5555FF'>/</font>cmd_line_parser.h<font color='#5555FF'>&gt;</font>
<font color='#0000FF'>#include</font> <font color='#5555FF'>&lt;</font>dlib<font color='#5555FF'>/</font>data_io.h<font color='#5555FF'>&gt;</font>
<font color='#0000FF'>#include</font> <font color='#5555FF'>&lt;</font>dlib<font color='#5555FF'>/</font>dnn.h<font color='#5555FF'>&gt;</font>
<font color='#0000FF'>#include</font> <font color='#5555FF'>&lt;</font>dlib<font color='#5555FF'>/</font>global_optimization.h<font color='#5555FF'>&gt;</font>
<font color='#0000FF'>#include</font> <font color='#5555FF'>&lt;</font>dlib<font color='#5555FF'>/</font>gui_widgets.h<font color='#5555FF'>&gt;</font>
<font color='#0000FF'>#include</font> <font color='#5555FF'>&lt;</font>dlib<font color='#5555FF'>/</font>svm_threaded.h<font color='#5555FF'>&gt;</font>

<font color='#0000FF'>using</font> <font color='#0000FF'>namespace</font> std;
<font color='#0000FF'>using</font> <font color='#0000FF'>namespace</font> dlib;

<font color='#009900'>// A custom definition of ResNet50 with a downsampling factor of 8 instead of 32.
</font><font color='#009900'>// It is essentially the original ResNet50, but without the max pooling and a
</font><font color='#009900'>// convolutional layer with a stride of 1 instead of 2 at the input.
</font><font color='#0000FF'>namespace</font> resnet50
<b>{</b>
    <font color='#0000FF'>using</font> <font color='#0000FF'>namespace</font> dlib;
    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font><font color='#5555FF'>&gt;</font> <font color='#0000FF'>class</font> <b><a name='BN'></a>BN</b><font color='#5555FF'>&gt;</font>
    <font color='#0000FF'>struct</font> <b><a name='def'></a>def</b>
    <b>{</b>
        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'><u>long</u></font> N, <font color='#0000FF'><u>int</u></font> K, <font color='#0000FF'><u>int</u></font> S, <font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font>
        <font color='#0000FF'>using</font> conv <font color='#5555FF'>=</font> add_layer<font color='#5555FF'>&lt;</font>con_<font color='#5555FF'>&lt;</font>N, K, K, S, S, K <font color='#5555FF'>/</font> <font color='#979000'>2</font>, K <font color='#5555FF'>/</font> <font color='#979000'>2</font><font color='#5555FF'>&gt;</font>, SUBNET<font color='#5555FF'>&gt;</font>;

        <font color='#0000FF'>template</font><font color='#5555FF'>&lt;</font><font color='#0000FF'><u>long</u></font> N, <font color='#0000FF'><u>int</u></font> S, <font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font>
        <font color='#0000FF'>using</font> bottleneck <font color='#5555FF'>=</font> BN<font color='#5555FF'>&lt;</font>conv<font color='#5555FF'>&lt;</font><font color='#979000'>4</font> <font color='#5555FF'>*</font> N, <font color='#979000'>1</font>, <font color='#979000'>1</font>, relu<font color='#5555FF'>&lt;</font>BN<font color='#5555FF'>&lt;</font>conv<font color='#5555FF'>&lt;</font>N, <font color='#979000'>3</font>, S, relu<font color='#5555FF'>&lt;</font>BN<font color='#5555FF'>&lt;</font>conv<font color='#5555FF'>&lt;</font>N, <font color='#979000'>1</font>, <font color='#979000'>1</font>, SUBNET<font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font>;

        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'><u>long</u></font> N,  <font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font>
        <font color='#0000FF'>using</font> residual <font color='#5555FF'>=</font> add_prev1<font color='#5555FF'>&lt;</font>bottleneck<font color='#5555FF'>&lt;</font>N, <font color='#979000'>1</font>, tag1<font color='#5555FF'>&lt;</font>SUBNET<font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font>;

        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font> <font color='#0000FF'>using</font> res_512 <font color='#5555FF'>=</font> relu<font color='#5555FF'>&lt;</font>residual<font color='#5555FF'>&lt;</font><font color='#979000'>512</font>, SUBNET<font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font>;
        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font> <font color='#0000FF'>using</font> res_256 <font color='#5555FF'>=</font> relu<font color='#5555FF'>&lt;</font>residual<font color='#5555FF'>&lt;</font><font color='#979000'>256</font>, SUBNET<font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font>;
        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font> <font color='#0000FF'>using</font> res_128 <font color='#5555FF'>=</font> relu<font color='#5555FF'>&lt;</font>residual<font color='#5555FF'>&lt;</font><font color='#979000'>128</font>, SUBNET<font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font>;
        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font> <font color='#0000FF'>using</font> res_64  <font color='#5555FF'>=</font> relu<font color='#5555FF'>&lt;</font>residual<font color='#5555FF'>&lt;</font><font color='#979000'>64</font>, SUBNET<font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font>;

        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'><u>long</u></font> N, <font color='#0000FF'><u>int</u></font> S, <font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font>
        <font color='#0000FF'>using</font> transition <font color='#5555FF'>=</font> add_prev2<font color='#5555FF'>&lt;</font>BN<font color='#5555FF'>&lt;</font>conv<font color='#5555FF'>&lt;</font><font color='#979000'>4</font> <font color='#5555FF'>*</font> N, <font color='#979000'>1</font>, S, skip1<font color='#5555FF'>&lt;</font>tag2<font color='#5555FF'>&lt;</font>bottleneck<font color='#5555FF'>&lt;</font>N, S, tag1<font color='#5555FF'>&lt;</font>SUBNET<font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font>;

        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> INPUT<font color='#5555FF'>&gt;</font>
        <font color='#0000FF'>using</font> backbone <font color='#5555FF'>=</font> avg_pool_everything<font color='#5555FF'>&lt;</font>
            repeat<font color='#5555FF'>&lt;</font><font color='#979000'>2</font>, res_512, transition<font color='#5555FF'>&lt;</font><font color='#979000'>512</font>, <font color='#979000'>2</font>,
            repeat<font color='#5555FF'>&lt;</font><font color='#979000'>5</font>, res_256, transition<font color='#5555FF'>&lt;</font><font color='#979000'>256</font>, <font color='#979000'>2</font>,
            repeat<font color='#5555FF'>&lt;</font><font color='#979000'>3</font>, res_128, transition<font color='#5555FF'>&lt;</font><font color='#979000'>128</font>, <font color='#979000'>2</font>,
            repeat<font color='#5555FF'>&lt;</font><font color='#979000'>2</font>, res_64,  transition<font color='#5555FF'>&lt;</font><font color='#979000'>64</font>, <font color='#979000'>1</font>,
            relu<font color='#5555FF'>&lt;</font>BN<font color='#5555FF'>&lt;</font>conv<font color='#5555FF'>&lt;</font><font color='#979000'>64</font>, <font color='#979000'>3</font>, <font color='#979000'>1</font>,INPUT<font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font>;
    <b>}</b>;
<b>}</b>;

<font color='#009900'>// This model namespace contains the definitions for:
</font><font color='#009900'>// - SSL model using the Barlow Twins loss, a projector head and an input_rgb_image_pair.
</font><font color='#009900'>// - A feature extractor model using the loss_metric (to get the outputs) and an input_rgb_image.
</font><font color='#0000FF'>namespace</font> model
<b>{</b>
    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font> <font color='#0000FF'>using</font> projector <font color='#5555FF'>=</font> fc<font color='#5555FF'>&lt;</font><font color='#979000'>128</font>, relu<font color='#5555FF'>&lt;</font>bn_fc<font color='#5555FF'>&lt;</font>fc<font color='#5555FF'>&lt;</font><font color='#979000'>512</font>, SUBNET<font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font>;
    <font color='#0000FF'>using</font> train <font color='#5555FF'>=</font> loss_barlow_twins<font color='#5555FF'>&lt;</font>projector<font color='#5555FF'>&lt;</font>resnet50::def<font color='#5555FF'>&lt;</font>bn_con<font color='#5555FF'>&gt;</font>::backbone<font color='#5555FF'>&lt;</font>input_rgb_image_pair<font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font>;
    <font color='#0000FF'>using</font> feats <font color='#5555FF'>=</font> loss_metric<font color='#5555FF'>&lt;</font>resnet50::def<font color='#5555FF'>&lt;</font>affine<font color='#5555FF'>&gt;</font>::backbone<font color='#5555FF'>&lt;</font>input_rgb_image<font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font>;
<b>}</b>

rectangle <b><a name='make_random_cropping_rect'></a>make_random_cropping_rect</b><font face='Lucida Console'>(</font>
    <font color='#0000FF'>const</font> matrix<font color='#5555FF'>&lt;</font>rgb_pixel<font color='#5555FF'>&gt;</font><font color='#5555FF'>&amp;</font> image,
    dlib::rand<font color='#5555FF'>&amp;</font> rnd
<font face='Lucida Console'>)</font>
<b>{</b>
    <font color='#0000FF'>const</font> <font color='#0000FF'><u>double</u></font> mins <font color='#5555FF'>=</font> <font color='#979000'>7.</font> <font color='#5555FF'>/</font> <font color='#979000'>15.</font>;
    <font color='#0000FF'>const</font> <font color='#0000FF'><u>double</u></font> maxs <font color='#5555FF'>=</font> <font color='#979000'>7.</font> <font color='#5555FF'>/</font> <font color='#979000'>8.</font>;
    <font color='#0000FF'>const</font> <font color='#0000FF'>auto</font> scale <font color='#5555FF'>=</font> rnd.<font color='#BB00BB'>get_double_in_range</font><font face='Lucida Console'>(</font>mins, maxs<font face='Lucida Console'>)</font>;
    <font color='#0000FF'>const</font> <font color='#0000FF'>auto</font> size <font color='#5555FF'>=</font> scale <font color='#5555FF'>*</font> std::<font color='#BB00BB'>min</font><font face='Lucida Console'>(</font>image.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>, image.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
    <font color='#0000FF'>const</font> rectangle <font color='#BB00BB'>rect</font><font face='Lucida Console'>(</font>size, size<font face='Lucida Console'>)</font>;
    <font color='#0000FF'>const</font> point <font color='#BB00BB'>offset</font><font face='Lucida Console'>(</font>rnd.<font color='#BB00BB'>get_random_32bit_number</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>%</font> <font face='Lucida Console'>(</font>image.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>-</font> rect.<font color='#BB00BB'>width</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>,
                       rnd.<font color='#BB00BB'>get_random_32bit_number</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>%</font> <font face='Lucida Console'>(</font>image.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>-</font> rect.<font color='#BB00BB'>height</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
    <font color='#0000FF'>return</font> <font color='#BB00BB'>move_rect</font><font face='Lucida Console'>(</font>rect, offset<font face='Lucida Console'>)</font>;
<b>}</b>

matrix<font color='#5555FF'>&lt;</font>rgb_pixel<font color='#5555FF'>&gt;</font> <b><a name='augment'></a>augment</b><font face='Lucida Console'>(</font>
    <font color='#0000FF'>const</font> matrix<font color='#5555FF'>&lt;</font>rgb_pixel<font color='#5555FF'>&gt;</font><font color='#5555FF'>&amp;</font> image,
    <font color='#0000FF'>const</font> <font color='#0000FF'><u>bool</u></font> prime,
    dlib::rand<font color='#5555FF'>&amp;</font> rnd
<font face='Lucida Console'>)</font>
<b>{</b>
    matrix<font color='#5555FF'>&lt;</font>rgb_pixel<font color='#5555FF'>&gt;</font> crop;
    <font color='#009900'>// blur
</font>    matrix<font color='#5555FF'>&lt;</font>rgb_pixel<font color='#5555FF'>&gt;</font> blurred;
    <font color='#0000FF'>const</font> <font color='#0000FF'><u>double</u></font> sigma <font color='#5555FF'>=</font> rnd.<font color='#BB00BB'>get_double_in_range</font><font face='Lucida Console'>(</font><font color='#979000'>0.1</font>, <font color='#979000'>1.1</font><font face='Lucida Console'>)</font>;
    <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font><font color='#5555FF'>!</font>prime <font color='#5555FF'>|</font><font color='#5555FF'>|</font> <font face='Lucida Console'>(</font>prime <font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font> rnd.<font color='#BB00BB'>get_random_double</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> <font color='#979000'>0.1</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>
    <b>{</b>
        <font color='#0000FF'>const</font> <font color='#0000FF'>auto</font> rect <font color='#5555FF'>=</font> <font color='#BB00BB'>gaussian_blur</font><font face='Lucida Console'>(</font>image, blurred, sigma<font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>extract_image_chip</font><font face='Lucida Console'>(</font>blurred, rect, crop<font face='Lucida Console'>)</font>;
        blurred <font color='#5555FF'>=</font> crop;
    <b>}</b>
    <font color='#0000FF'>else</font>
    <b>{</b>
        blurred <font color='#5555FF'>=</font> image;
    <b>}</b>

    <font color='#009900'>// randomly crop
</font>    <font color='#0000FF'>const</font> <font color='#0000FF'>auto</font> rect <font color='#5555FF'>=</font> <font color='#BB00BB'>make_random_cropping_rect</font><font face='Lucida Console'>(</font>image, rnd<font face='Lucida Console'>)</font>;
    <font color='#BB00BB'>extract_image_chip</font><font face='Lucida Console'>(</font>blurred, <font color='#BB00BB'>chip_details</font><font face='Lucida Console'>(</font>rect, <font color='#BB00BB'>chip_dims</font><font face='Lucida Console'>(</font><font color='#979000'>32</font>, <font color='#979000'>32</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>, crop<font face='Lucida Console'>)</font>;

    <font color='#009900'>// image left-right flip
</font>    <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>rnd.<font color='#BB00BB'>get_random_double</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> <font color='#979000'>0.5</font><font face='Lucida Console'>)</font>
        <font color='#BB00BB'>flip_image_left_right</font><font face='Lucida Console'>(</font>crop<font face='Lucida Console'>)</font>;

    <font color='#009900'>// color augmentation
</font>    <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>rnd.<font color='#BB00BB'>get_random_double</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> <font color='#979000'>0.8</font><font face='Lucida Console'>)</font>
        <font color='#BB00BB'>disturb_colors</font><font face='Lucida Console'>(</font>crop, rnd, <font color='#979000'>0.5</font>, <font color='#979000'>0.5</font><font face='Lucida Console'>)</font>;

    <font color='#009900'>// grayscale
</font>    <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>rnd.<font color='#BB00BB'>get_random_double</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> <font color='#979000'>0.2</font><font face='Lucida Console'>)</font>
    <b>{</b>
        matrix<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>unsigned</u></font> <font color='#0000FF'><u>char</u></font><font color='#5555FF'>&gt;</font> gray;
        <font color='#BB00BB'>assign_image</font><font face='Lucida Console'>(</font>gray, crop<font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>assign_image</font><font face='Lucida Console'>(</font>crop, gray<font face='Lucida Console'>)</font>;
    <b>}</b>

    <font color='#009900'>// solarize
</font>    <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>prime <font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font> rnd.<font color='#BB00BB'>get_random_double</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> <font color='#979000'>0.2</font><font face='Lucida Console'>)</font>
    <b>{</b>
        <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'>auto</font><font color='#5555FF'>&amp;</font> p : crop<font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>p.red <font color='#5555FF'>&gt;</font> <font color='#979000'>128</font><font face='Lucida Console'>)</font>
                p.red <font color='#5555FF'>=</font> <font color='#979000'>255</font> <font color='#5555FF'>-</font> p.red;
            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>p.green <font color='#5555FF'>&gt;</font> <font color='#979000'>128</font><font face='Lucida Console'>)</font>
                p.green <font color='#5555FF'>=</font> <font color='#979000'>255</font> <font color='#5555FF'>-</font> p.green;
            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>p.blue <font color='#5555FF'>&gt;</font> <font color='#979000'>128</font><font face='Lucida Console'>)</font>
                p.blue <font color='#5555FF'>=</font> <font color='#979000'>255</font> <font color='#5555FF'>-</font> p.blue;
        <b>}</b>
    <b>}</b>
    <font color='#0000FF'>return</font> crop;
<b>}</b>

<font color='#0000FF'><u>int</u></font> <b><a name='main'></a>main</b><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> <font color='#0000FF'><u>int</u></font> argc, <font color='#0000FF'>const</font> <font color='#0000FF'><u>char</u></font><font color='#5555FF'>*</font><font color='#5555FF'>*</font> argv<font face='Lucida Console'>)</font>
<font color='#0000FF'>try</font>
<b>{</b>
    <font color='#009900'>// The default settings are fine for the example already.
</font>    command_line_parser parser;
    parser.<font color='#BB00BB'>add_option</font><font face='Lucida Console'>(</font>"<font color='#CC0000'>batch</font>", "<font color='#CC0000'>set the mini batch size per GPU (default: 64)</font>", <font color='#979000'>1</font><font face='Lucida Console'>)</font>;
    parser.<font color='#BB00BB'>add_option</font><font face='Lucida Console'>(</font>"<font color='#CC0000'>dims</font>", "<font color='#CC0000'>set the projector dimensions (default: 128)</font>", <font color='#979000'>1</font><font face='Lucida Console'>)</font>;
    parser.<font color='#BB00BB'>add_option</font><font face='Lucida Console'>(</font>"<font color='#CC0000'>lambda</font>", "<font color='#CC0000'>penalize off-diagonal terms (default: 1/dims)</font>", <font color='#979000'>1</font><font face='Lucida Console'>)</font>;
    parser.<font color='#BB00BB'>add_option</font><font face='Lucida Console'>(</font>"<font color='#CC0000'>learning-rate</font>", "<font color='#CC0000'>set the initial learning rate (default: 1e-3)</font>", <font color='#979000'>1</font><font face='Lucida Console'>)</font>;
    parser.<font color='#BB00BB'>add_option</font><font face='Lucida Console'>(</font>"<font color='#CC0000'>min-learning-rate</font>", "<font color='#CC0000'>set the min learning rate (default: 1e-5)</font>", <font color='#979000'>1</font><font face='Lucida Console'>)</font>;
    parser.<font color='#BB00BB'>add_option</font><font face='Lucida Console'>(</font>"<font color='#CC0000'>num-gpus</font>", "<font color='#CC0000'>number of GPUs (default: 1)</font>", <font color='#979000'>1</font><font face='Lucida Console'>)</font>;
    parser.<font color='#BB00BB'>set_group_name</font><font face='Lucida Console'>(</font>"<font color='#CC0000'>Help Options</font>"<font face='Lucida Console'>)</font>;
    parser.<font color='#BB00BB'>add_option</font><font face='Lucida Console'>(</font>"<font color='#CC0000'>h</font>", "<font color='#CC0000'>alias for --help</font>"<font face='Lucida Console'>)</font>;
    parser.<font color='#BB00BB'>add_option</font><font face='Lucida Console'>(</font>"<font color='#CC0000'>help</font>", "<font color='#CC0000'>display this message and exit</font>"<font face='Lucida Console'>)</font>;
    parser.<font color='#BB00BB'>parse</font><font face='Lucida Console'>(</font>argc, argv<font face='Lucida Console'>)</font>;

    <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>parser.<font color='#BB00BB'>number_of_arguments</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> <font color='#979000'>1</font> <font color='#5555FF'>|</font><font color='#5555FF'>|</font> parser.<font color='#BB00BB'>option</font><font face='Lucida Console'>(</font>"<font color='#CC0000'>h</font>"<font face='Lucida Console'>)</font> <font color='#5555FF'>|</font><font color='#5555FF'>|</font> parser.<font color='#BB00BB'>option</font><font face='Lucida Console'>(</font>"<font color='#CC0000'>help</font>"<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>
    <b>{</b>
        cout <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>This example needs the CIFAR-10 dataset to run.</font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> endl;
        cout <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>You can get CIFAR-10 from https://www.cs.toronto.edu/~kriz/cifar.html</font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> endl;
        cout <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>Download the binary version the dataset, decompress it, and put the 6</font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> endl;
        cout <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>bin files in a folder.  Then give that folder as input to this program.</font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> endl;
        parser.<font color='#BB00BB'>print_options</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
        <font color='#0000FF'>return</font> EXIT_SUCCESS;
    <b>}</b>

    <font color='#0000FF'>const</font> <font color='#0000FF'><u>size_t</u></font> num_gpus <font color='#5555FF'>=</font> <font color='#BB00BB'>get_option</font><font face='Lucida Console'>(</font>parser, "<font color='#CC0000'>num-gpus</font>", <font color='#979000'>1</font><font face='Lucida Console'>)</font>;
    <font color='#0000FF'>const</font> <font color='#0000FF'><u>size_t</u></font> batch_size <font color='#5555FF'>=</font> <font color='#BB00BB'>get_option</font><font face='Lucida Console'>(</font>parser, "<font color='#CC0000'>batch</font>", <font color='#979000'>64</font><font face='Lucida Console'>)</font> <font color='#5555FF'>*</font> num_gpus;
    <font color='#0000FF'>const</font> <font color='#0000FF'><u>long</u></font> dims <font color='#5555FF'>=</font> <font color='#BB00BB'>get_option</font><font face='Lucida Console'>(</font>parser, "<font color='#CC0000'>dims</font>", <font color='#979000'>128</font><font face='Lucida Console'>)</font>;
    <font color='#0000FF'>const</font> <font color='#0000FF'><u>double</u></font> lambda <font color='#5555FF'>=</font> <font color='#BB00BB'>get_option</font><font face='Lucida Console'>(</font>parser, "<font color='#CC0000'>lambda</font>", <font color='#979000'>1.0</font> <font color='#5555FF'>/</font> dims<font face='Lucida Console'>)</font>;
    <font color='#0000FF'>const</font> <font color='#0000FF'><u>double</u></font> learning_rate <font color='#5555FF'>=</font> <font color='#BB00BB'>get_option</font><font face='Lucida Console'>(</font>parser, "<font color='#CC0000'>learning-rate</font>", <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>3</font><font face='Lucida Console'>)</font>;
    <font color='#0000FF'>const</font> <font color='#0000FF'><u>double</u></font> min_learning_rate <font color='#5555FF'>=</font> <font color='#BB00BB'>get_option</font><font face='Lucida Console'>(</font>parser, "<font color='#CC0000'>min-learning-rate</font>", <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>5</font><font face='Lucida Console'>)</font>;

    <font color='#009900'>// Load the CIFAR-10 dataset into memory.
</font>    std::vector<font color='#5555FF'>&lt;</font>matrix<font color='#5555FF'>&lt;</font>rgb_pixel<font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font> training_images, testing_images;
    std::vector<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>unsigned</u></font> <font color='#0000FF'><u>long</u></font><font color='#5555FF'>&gt;</font> training_labels, testing_labels;
    <font color='#BB00BB'>load_cifar_10_dataset</font><font face='Lucida Console'>(</font>parser[<font color='#979000'>0</font>], training_images, training_labels, testing_images, testing_labels<font face='Lucida Console'>)</font>;

    <font color='#009900'>// Initialize the model with the specified projector dimensions and lambda.  According to the
</font>    <font color='#009900'>// second paper, lambda = 1/dims works well on CIFAR-10.
</font>    model::train <font color='#BB00BB'>net</font><font face='Lucida Console'>(</font><font face='Lucida Console'>(</font><font color='#BB00BB'>loss_barlow_twins_</font><font face='Lucida Console'>(</font>lambda<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
    layer<font color='#5555FF'>&lt;</font><font color='#979000'>1</font><font color='#5555FF'>&gt;</font><font face='Lucida Console'>(</font>net<font face='Lucida Console'>)</font>.<font color='#BB00BB'>layer_details</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>.<font color='#BB00BB'>set_num_outputs</font><font face='Lucida Console'>(</font>dims<font face='Lucida Console'>)</font>;
    <font color='#BB00BB'>disable_duplicative_biases</font><font face='Lucida Console'>(</font>net<font face='Lucida Console'>)</font>;
    dlib::rand rnd;
    std::vector<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>int</u></font><font color='#5555FF'>&gt;</font> <font color='#BB00BB'>gpus</font><font face='Lucida Console'>(</font>num_gpus<font face='Lucida Console'>)</font>;
    std::<font color='#BB00BB'>iota</font><font face='Lucida Console'>(</font>gpus.<font color='#BB00BB'>begin</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>, gpus.<font color='#BB00BB'>end</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>, <font color='#979000'>0</font><font face='Lucida Console'>)</font>;

    <font color='#009900'>// Train the feature extractor using the Barlow Twins method
</font>    <b>{</b>
        dnn_trainer<font color='#5555FF'>&lt;</font>model::train, adam<font color='#5555FF'>&gt;</font> <font color='#BB00BB'>trainer</font><font face='Lucida Console'>(</font>net, <font color='#BB00BB'>adam</font><font face='Lucida Console'>(</font><font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>6</font>, <font color='#979000'>0.9</font>, <font color='#979000'>0.999</font><font face='Lucida Console'>)</font>, gpus<font face='Lucida Console'>)</font>;
        trainer.<font color='#BB00BB'>set_mini_batch_size</font><font face='Lucida Console'>(</font>batch_size<font face='Lucida Console'>)</font>;
        trainer.<font color='#BB00BB'>set_learning_rate</font><font face='Lucida Console'>(</font>learning_rate<font face='Lucida Console'>)</font>;
        trainer.<font color='#BB00BB'>set_min_learning_rate</font><font face='Lucida Console'>(</font>min_learning_rate<font face='Lucida Console'>)</font>;
        trainer.<font color='#BB00BB'>set_iterations_without_progress_threshold</font><font face='Lucida Console'>(</font><font color='#979000'>10000</font><font face='Lucida Console'>)</font>;
        trainer.<font color='#BB00BB'>set_synchronization_file</font><font face='Lucida Console'>(</font>"<font color='#CC0000'>barlow_twins_sync</font>"<font face='Lucida Console'>)</font>;
        trainer.<font color='#BB00BB'>be_verbose</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
        cout <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> trainer <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> endl;

        <font color='#009900'>// During the training, we will compute the empirical cross-correlation matrix
</font>        <font color='#009900'>// between the features of both versions of the augmented images.  This matrix
</font>        <font color='#009900'>// should be getting close to the identity matrix as the training progresses.
</font>        <font color='#009900'>// Note that this step is already done in the loss layer, and it's not necessary
</font>        <font color='#009900'>// to do it here for the example to work.  However, it provides a nice
</font>        <font color='#009900'>// visualization of the training progress: the closer to the identity matrix,
</font>        <font color='#009900'>// the better.
</font>        resizable_tensor eccm;
        eccm.<font color='#BB00BB'>set_size</font><font face='Lucida Console'>(</font>dims, dims<font face='Lucida Console'>)</font>;
        <font color='#009900'>// Some tensors needed to perform batch normalization
</font>        resizable_tensor za_norm, zb_norm, means, invstds, rms, rvs, gamma, beta;
        <font color='#0000FF'>const</font> <font color='#0000FF'><u>double</u></font> eps <font color='#5555FF'>=</font> DEFAULT_BATCH_NORM_EPS;
        gamma.<font color='#BB00BB'>set_size</font><font face='Lucida Console'>(</font><font color='#979000'>1</font>, dims<font face='Lucida Console'>)</font>;
        beta.<font color='#BB00BB'>set_size</font><font face='Lucida Console'>(</font><font color='#979000'>1</font>, dims<font face='Lucida Console'>)</font>;
        image_window win;

        std::vector<font color='#5555FF'>&lt;</font>std::pair<font color='#5555FF'>&lt;</font>matrix<font color='#5555FF'>&lt;</font>rgb_pixel<font color='#5555FF'>&gt;</font>, matrix<font color='#5555FF'>&lt;</font>rgb_pixel<font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font> batch;
        <font color='#0000FF'>while</font> <font face='Lucida Console'>(</font>trainer.<font color='#BB00BB'>get_learning_rate</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&gt;</font><font color='#5555FF'>=</font> trainer.<font color='#BB00BB'>get_min_learning_rate</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>
        <b>{</b>
            batch.<font color='#BB00BB'>clear</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'>while</font> <font face='Lucida Console'>(</font>batch.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> trainer.<font color='#BB00BB'>get_mini_batch_size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>
            <b>{</b>
                <font color='#0000FF'>const</font> <font color='#0000FF'>auto</font> idx <font color='#5555FF'>=</font> rnd.<font color='#BB00BB'>get_random_32bit_number</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>%</font> training_images.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
                <font color='#0000FF'>auto</font> image <font color='#5555FF'>=</font> training_images[idx];
                batch.<font color='#BB00BB'>emplace_back</font><font face='Lucida Console'>(</font><font color='#BB00BB'>augment</font><font face='Lucida Console'>(</font>image, <font color='#979000'>false</font>, rnd<font face='Lucida Console'>)</font>, <font color='#BB00BB'>augment</font><font face='Lucida Console'>(</font>image, <font color='#979000'>true</font>, rnd<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
            <b>}</b>
            trainer.<font color='#BB00BB'>train_one_step</font><font face='Lucida Console'>(</font>batch<font face='Lucida Console'>)</font>;

            <font color='#009900'>// Compute the empirical cross-correlation matrix every 100 steps. Again,
</font>            <font color='#009900'>// this is not needed for the training to work, but it's nice to visualize.
</font>            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>trainer.<font color='#BB00BB'>get_train_one_step_calls</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>%</font> <font color='#979000'>100</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>0</font><font face='Lucida Console'>)</font>
            <b>{</b>
                <font color='#009900'>// Wait for threaded processing to stop in the trainer.
</font>                trainer.<font color='#BB00BB'>get_net</font><font face='Lucida Console'>(</font>force_flush_to_disk::no<font face='Lucida Console'>)</font>;
                <font color='#009900'>// Get the output from the last fc layer
</font>                <font color='#0000FF'>const</font> <font color='#0000FF'>auto</font><font color='#5555FF'>&amp;</font> out <font color='#5555FF'>=</font> net.<font color='#BB00BB'>subnet</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>.<font color='#BB00BB'>get_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
                <font color='#009900'>// The trainer might have synchronized its state to the disk and cleaned
</font>                <font color='#009900'>// the network state. If that happens, the output will be empty, in
</font>                <font color='#009900'>// which case, we just skip the empirical cross-correlation matrix
</font>                <font color='#009900'>// computation.
</font>                <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>out.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>0</font><font face='Lucida Console'>)</font>
                    <font color='#0000FF'>continue</font>;
                <font color='#009900'>// Separate both augmented versions of the images
</font>                alias_tensor <font color='#BB00BB'>split</font><font face='Lucida Console'>(</font>out.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>/</font> <font color='#979000'>2</font>, dims<font face='Lucida Console'>)</font>;
                <font color='#0000FF'>auto</font> za <font color='#5555FF'>=</font> <font color='#BB00BB'>split</font><font face='Lucida Console'>(</font>out<font face='Lucida Console'>)</font>;
                <font color='#0000FF'>auto</font> zb <font color='#5555FF'>=</font> <font color='#BB00BB'>split</font><font face='Lucida Console'>(</font>out, split.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
                gamma <font color='#5555FF'>=</font> <font color='#979000'>1</font>;
                beta <font color='#5555FF'>=</font> <font color='#979000'>0</font>;
                <font color='#009900'>// Perform batch normalization on each feature representation, independently.
</font>                tt::<font color='#BB00BB'>batch_normalize</font><font face='Lucida Console'>(</font>eps, za_norm, means, invstds, <font color='#979000'>1</font>, rms, rvs, za, gamma, beta<font face='Lucida Console'>)</font>;
                tt::<font color='#BB00BB'>batch_normalize</font><font face='Lucida Console'>(</font>eps, zb_norm, means, invstds, <font color='#979000'>1</font>, rms, rvs, za, gamma, beta<font face='Lucida Console'>)</font>;
                <font color='#009900'>// Compute the empirical cross-correlation matrix between the features and
</font>                <font color='#009900'>// visualize it.
</font>                tt::<font color='#BB00BB'>gemm</font><font face='Lucida Console'>(</font><font color='#979000'>0</font>, eccm, <font color='#979000'>1</font>, za_norm, <font color='#979000'>true</font>, zb_norm, <font color='#979000'>false</font><font face='Lucida Console'>)</font>;
                eccm <font color='#5555FF'>/</font><font color='#5555FF'>=</font> batch_size;
                win.<font color='#BB00BB'>set_image</font><font face='Lucida Console'>(</font><font color='#BB00BB'>round</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>eccm<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <font color='#5555FF'>*</font> <font color='#979000'>255</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
                win.<font color='#BB00BB'>set_title</font><font face='Lucida Console'>(</font>"<font color='#CC0000'>Barlow Twins step#: </font>" <font color='#5555FF'>+</font> <font color='#BB00BB'>to_string</font><font face='Lucida Console'>(</font>trainer.<font color='#BB00BB'>get_train_one_step_calls</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
            <b>}</b>
        <b>}</b>
        trainer.<font color='#BB00BB'>get_net</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
        net.<font color='#BB00BB'>clean</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
        <font color='#009900'>// After training, we can discard the projector head and just keep the backone
</font>        <font color='#009900'>// to train it or finetune it on other downstream tasks.
</font>        <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>"<font color='#CC0000'>resnet50_self_supervised_cifar_10.net</font>"<font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> layer<font color='#5555FF'>&lt;</font><font color='#979000'>5</font><font color='#5555FF'>&gt;</font><font face='Lucida Console'>(</font>net<font face='Lucida Console'>)</font>;
    <b>}</b>

    <font color='#009900'>// Now, we initialize the feature extractor model with the backbone we have just learned.
</font>    model::feats <font color='#BB00BB'>fnet</font><font face='Lucida Console'>(</font>layer<font color='#5555FF'>&lt;</font><font color='#979000'>5</font><font color='#5555FF'>&gt;</font><font face='Lucida Console'>(</font>net<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
    <font color='#009900'>// And we will generate all the features for the training set to train a multiclass SVM
</font>    <font color='#009900'>// classifier.
</font>    std::vector<font color='#5555FF'>&lt;</font>matrix<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>float</u></font>, <font color='#979000'>0</font>, <font color='#979000'>1</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font> features;
    cout <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>Extracting features for linear classifier...</font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> endl;
    features <font color='#5555FF'>=</font> <font color='#BB00BB'>fnet</font><font face='Lucida Console'>(</font>training_images, <font color='#979000'>4</font> <font color='#5555FF'>*</font> batch_size<font face='Lucida Console'>)</font>;
    vector_normalizer<font color='#5555FF'>&lt;</font>matrix<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>float</u></font>, <font color='#979000'>0</font>, <font color='#979000'>1</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font> normalizer;
    normalizer.<font color='#BB00BB'>train</font><font face='Lucida Console'>(</font>features<font face='Lucida Console'>)</font>;
    <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'>auto</font><font color='#5555FF'>&amp;</font> feature : features<font face='Lucida Console'>)</font>
        feature <font color='#5555FF'>=</font> <font color='#BB00BB'>normalizer</font><font face='Lucida Console'>(</font>feature<font face='Lucida Console'>)</font>;

    <font color='#009900'>// Find the most appropriate C setting using find_max_global.
</font>    <font color='#0000FF'>auto</font> cross_validation_score <font color='#5555FF'>=</font> [<font color='#5555FF'>&amp;</font>]<font face='Lucida Console'>(</font><font color='#0000FF'>const</font> <font color='#0000FF'><u>double</u></font> c<font face='Lucida Console'>)</font>
    <b>{</b>
        svm_multiclass_linear_trainer<font color='#5555FF'>&lt;</font>linear_kernel<font color='#5555FF'>&lt;</font>matrix<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>float</u></font>, <font color='#979000'>0</font>, <font color='#979000'>1</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font>, <font color='#0000FF'><u>unsigned</u></font> <font color='#0000FF'><u>long</u></font><font color='#5555FF'>&gt;</font> trainer;
        trainer.<font color='#BB00BB'>set_c</font><font face='Lucida Console'>(</font>c<font face='Lucida Console'>)</font>;
        trainer.<font color='#BB00BB'>set_epsilon</font><font face='Lucida Console'>(</font><font color='#979000'>0.01</font><font face='Lucida Console'>)</font>;
        trainer.<font color='#BB00BB'>set_max_iterations</font><font face='Lucida Console'>(</font><font color='#979000'>100</font><font face='Lucida Console'>)</font>;
        trainer.<font color='#BB00BB'>set_num_threads</font><font face='Lucida Console'>(</font>std::thread::<font color='#BB00BB'>hardware_concurrency</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
        cout <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>C: </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> c <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> endl;
        <font color='#0000FF'>const</font> <font color='#0000FF'>auto</font> cm <font color='#5555FF'>=</font> <font color='#BB00BB'>cross_validate_multiclass_trainer</font><font face='Lucida Console'>(</font>trainer, features, training_labels, <font color='#979000'>3</font><font face='Lucida Console'>)</font>;
        <font color='#0000FF'>const</font> <font color='#0000FF'><u>double</u></font> accuracy <font color='#5555FF'>=</font> <font color='#BB00BB'>sum</font><font face='Lucida Console'>(</font><font color='#BB00BB'>diag</font><font face='Lucida Console'>(</font>cm<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <font color='#5555FF'>/</font> <font color='#BB00BB'>sum</font><font face='Lucida Console'>(</font>cm<font face='Lucida Console'>)</font>;
        cout <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>cross validation accuracy: </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> accuracy <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> endl;
        cout <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>confusion matrix:\n </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> cm <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> endl;
        <font color='#0000FF'>return</font> accuracy;
    <b>}</b>;
    <font color='#0000FF'>const</font> <font color='#0000FF'>auto</font> result <font color='#5555FF'>=</font> <font color='#BB00BB'>find_max_global</font><font face='Lucida Console'>(</font>cross_validation_score, <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>3</font>, <font color='#979000'>1000</font>, <font color='#BB00BB'>max_function_calls</font><font face='Lucida Console'>(</font><font color='#979000'>50</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
    cout <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>Best C: </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> result.<font color='#BB00BB'>x</font><font face='Lucida Console'>(</font><font color='#979000'>0</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> endl;

    <font color='#009900'>// Proceed to train the SVM classifier with the best C.
</font>    svm_multiclass_linear_trainer<font color='#5555FF'>&lt;</font>linear_kernel<font color='#5555FF'>&lt;</font>matrix<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>float</u></font>, <font color='#979000'>0</font>, <font color='#979000'>1</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font>, <font color='#0000FF'><u>unsigned</u></font> <font color='#0000FF'><u>long</u></font><font color='#5555FF'>&gt;</font> trainer;
    trainer.<font color='#BB00BB'>set_num_threads</font><font face='Lucida Console'>(</font>std::thread::<font color='#BB00BB'>hardware_concurrency</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
    trainer.<font color='#BB00BB'>set_c</font><font face='Lucida Console'>(</font>result.<font color='#BB00BB'>x</font><font face='Lucida Console'>(</font><font color='#979000'>0</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
    cout <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>Training Multiclass SVM...</font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> endl;
    <font color='#0000FF'>const</font> <font color='#0000FF'>auto</font> df <font color='#5555FF'>=</font> trainer.<font color='#BB00BB'>train</font><font face='Lucida Console'>(</font>features, training_labels<font face='Lucida Console'>)</font>;
    <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>"<font color='#CC0000'>multiclass_svm_cifar_10.dat</font>"<font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> df;

    <font color='#009900'>// Finally, we can compute the accuracy of the model on the CIFAR-10 train and test images.
</font>    <font color='#0000FF'>auto</font> compute_accuracy <font color='#5555FF'>=</font> [<font color='#5555FF'>&amp;</font>fnet, <font color='#5555FF'>&amp;</font>df, batch_size]<font face='Lucida Console'>(</font>
        <font color='#0000FF'>const</font> std::vector<font color='#5555FF'>&lt;</font>matrix<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>float</u></font>, <font color='#979000'>0</font>, <font color='#979000'>1</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&amp;</font> samples,
        <font color='#0000FF'>const</font> std::vector<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>unsigned</u></font> <font color='#0000FF'><u>long</u></font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&amp;</font> labels
    <font face='Lucida Console'>)</font>
    <b>{</b>
        <font color='#0000FF'><u>size_t</u></font> num_right <font color='#5555FF'>=</font> <font color='#979000'>0</font>;
        <font color='#0000FF'><u>size_t</u></font> num_wrong <font color='#5555FF'>=</font> <font color='#979000'>0</font>;
        <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>size_t</u></font> i <font color='#5555FF'>=</font> <font color='#979000'>0</font>; i <font color='#5555FF'>&lt;</font> labels.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>i<font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>labels[i] <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#BB00BB'>df</font><font face='Lucida Console'>(</font>samples[i]<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>
                <font color='#5555FF'>+</font><font color='#5555FF'>+</font>num_right;
            <font color='#0000FF'>else</font>
                <font color='#5555FF'>+</font><font color='#5555FF'>+</font>num_wrong;
        <b>}</b>
        cout <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>  num right:  </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> num_right <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> endl;
        cout <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>  num wrong:  </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> num_wrong <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> endl;
        cout <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>  accuracy:   </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> num_right <font color='#5555FF'>/</font> <font color='#0000FF'>static_cast</font><font color='#5555FF'>&lt;</font><font color='#0000FF'><u>double</u></font><font color='#5555FF'>&gt;</font><font face='Lucida Console'>(</font>num_right <font color='#5555FF'>+</font> num_wrong<font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> endl;
        cout <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>  error rate: </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> num_wrong <font color='#5555FF'>/</font> <font color='#0000FF'>static_cast</font><font color='#5555FF'>&lt;</font><font color='#0000FF'><u>double</u></font><font color='#5555FF'>&gt;</font><font face='Lucida Console'>(</font>num_right <font color='#5555FF'>+</font> num_wrong<font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> endl;
    <b>}</b>;

    <font color='#009900'>// We should get a training accuracy of around 93% and a testing accuracy of around 89%.
</font>    cout <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>\ntraining accuracy</font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> endl;
    <font color='#BB00BB'>compute_accuracy</font><font face='Lucida Console'>(</font>features, training_labels<font face='Lucida Console'>)</font>;
    cout <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>\ntesting accuracy</font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> endl;
    features <font color='#5555FF'>=</font> <font color='#BB00BB'>fnet</font><font face='Lucida Console'>(</font>testing_images, <font color='#979000'>4</font> <font color='#5555FF'>*</font> batch_size<font face='Lucida Console'>)</font>;
    <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'>auto</font><font color='#5555FF'>&amp;</font> feature : features<font face='Lucida Console'>)</font>
        feature <font color='#5555FF'>=</font> <font color='#BB00BB'>normalizer</font><font face='Lucida Console'>(</font>feature<font face='Lucida Console'>)</font>;
    <font color='#BB00BB'>compute_accuracy</font><font face='Lucida Console'>(</font>features, testing_labels<font face='Lucida Console'>)</font>;
    <font color='#0000FF'>return</font> EXIT_SUCCESS;
<b>}</b>
<font color='#0000FF'>catch</font> <font face='Lucida Console'>(</font><font color='#0000FF'>const</font> exception<font color='#5555FF'>&amp;</font> e<font face='Lucida Console'>)</font>
<b>{</b>
    cout <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> e.<font color='#BB00BB'>what</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> endl;
    <font color='#0000FF'>return</font> EXIT_FAILURE;
<b>}</b>

</pre></body></html>